<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">

<head>

<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="pandoc" />

<meta name="viewport" content="width=device-width, initial-scale=1">

<meta name="author" content="James McCreight" />

<meta name="date" content="2017-05-01" />

<title>Collect the SNODAS product and build a local database.</title>



<style type="text/css">code{white-space: pre;}</style>
<style type="text/css">
div.sourceCode { overflow-x: auto; }
table.sourceCode, tr.sourceCode, td.lineNumbers, td.sourceCode {
  margin: 0; padding: 0; vertical-align: baseline; border: none; }
table.sourceCode { width: 100%; line-height: 100%; }
td.lineNumbers { text-align: right; padding-right: 4px; padding-left: 4px; color: #aaaaaa; border-right: 1px solid #aaaaaa; }
td.sourceCode { padding-left: 5px; }
code > span.kw { color: #007020; font-weight: bold; } /* Keyword */
code > span.dt { color: #902000; } /* DataType */
code > span.dv { color: #40a070; } /* DecVal */
code > span.bn { color: #40a070; } /* BaseN */
code > span.fl { color: #40a070; } /* Float */
code > span.ch { color: #4070a0; } /* Char */
code > span.st { color: #4070a0; } /* String */
code > span.co { color: #60a0b0; font-style: italic; } /* Comment */
code > span.ot { color: #007020; } /* Other */
code > span.al { color: #ff0000; font-weight: bold; } /* Alert */
code > span.fu { color: #06287e; } /* Function */
code > span.er { color: #ff0000; font-weight: bold; } /* Error */
code > span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
code > span.cn { color: #880000; } /* Constant */
code > span.sc { color: #4070a0; } /* SpecialChar */
code > span.vs { color: #4070a0; } /* VerbatimString */
code > span.ss { color: #bb6688; } /* SpecialString */
code > span.im { } /* Import */
code > span.va { color: #19177c; } /* Variable */
code > span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
code > span.op { color: #666666; } /* Operator */
code > span.bu { } /* BuiltIn */
code > span.ex { } /* Extension */
code > span.pp { color: #bc7a00; } /* Preprocessor */
code > span.at { color: #7d9029; } /* Attribute */
code > span.do { color: #ba2121; font-style: italic; } /* Documentation */
code > span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
code > span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
code > span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
</style>



<link href="data:text/css;charset=utf-8,body%20%7B%0Abackground%2Dcolor%3A%20%23fff%3B%0Amargin%3A%201em%20auto%3B%0Amax%2Dwidth%3A%20700px%3B%0Aoverflow%3A%20visible%3B%0Apadding%2Dleft%3A%202em%3B%0Apadding%2Dright%3A%202em%3B%0Afont%2Dfamily%3A%20%22Open%20Sans%22%2C%20%22Helvetica%20Neue%22%2C%20Helvetica%2C%20Arial%2C%20sans%2Dserif%3B%0Afont%2Dsize%3A%2014px%3B%0Aline%2Dheight%3A%201%2E35%3B%0A%7D%0A%23header%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0A%23TOC%20%7B%0Aclear%3A%20both%3B%0Amargin%3A%200%200%2010px%2010px%3B%0Apadding%3A%204px%3B%0Awidth%3A%20400px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Aborder%2Dradius%3A%205px%3B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Afont%2Dsize%3A%2013px%3B%0Aline%2Dheight%3A%201%2E3%3B%0A%7D%0A%23TOC%20%2Etoctitle%20%7B%0Afont%2Dweight%3A%20bold%3B%0Afont%2Dsize%3A%2015px%3B%0Amargin%2Dleft%3A%205px%3B%0A%7D%0A%23TOC%20ul%20%7B%0Apadding%2Dleft%3A%2040px%3B%0Amargin%2Dleft%3A%20%2D1%2E5em%3B%0Amargin%2Dtop%3A%205px%3B%0Amargin%2Dbottom%3A%205px%3B%0A%7D%0A%23TOC%20ul%20ul%20%7B%0Amargin%2Dleft%3A%20%2D2em%3B%0A%7D%0A%23TOC%20li%20%7B%0Aline%2Dheight%3A%2016px%3B%0A%7D%0Atable%20%7B%0Amargin%3A%201em%20auto%3B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dcolor%3A%20%23DDDDDD%3B%0Aborder%2Dstyle%3A%20outset%3B%0Aborder%2Dcollapse%3A%20collapse%3B%0A%7D%0Atable%20th%20%7B%0Aborder%2Dwidth%3A%202px%3B%0Apadding%3A%205px%3B%0Aborder%2Dstyle%3A%20inset%3B%0A%7D%0Atable%20td%20%7B%0Aborder%2Dwidth%3A%201px%3B%0Aborder%2Dstyle%3A%20inset%3B%0Aline%2Dheight%3A%2018px%3B%0Apadding%3A%205px%205px%3B%0A%7D%0Atable%2C%20table%20th%2C%20table%20td%20%7B%0Aborder%2Dleft%2Dstyle%3A%20none%3B%0Aborder%2Dright%2Dstyle%3A%20none%3B%0A%7D%0Atable%20thead%2C%20table%20tr%2Eeven%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Ap%20%7B%0Amargin%3A%200%2E5em%200%3B%0A%7D%0Ablockquote%20%7B%0Abackground%2Dcolor%3A%20%23f6f6f6%3B%0Apadding%3A%200%2E25em%200%2E75em%3B%0A%7D%0Ahr%20%7B%0Aborder%2Dstyle%3A%20solid%3B%0Aborder%3A%20none%3B%0Aborder%2Dtop%3A%201px%20solid%20%23777%3B%0Amargin%3A%2028px%200%3B%0A%7D%0Adl%20%7B%0Amargin%2Dleft%3A%200%3B%0A%7D%0Adl%20dd%20%7B%0Amargin%2Dbottom%3A%2013px%3B%0Amargin%2Dleft%3A%2013px%3B%0A%7D%0Adl%20dt%20%7B%0Afont%2Dweight%3A%20bold%3B%0A%7D%0Aul%20%7B%0Amargin%2Dtop%3A%200%3B%0A%7D%0Aul%20li%20%7B%0Alist%2Dstyle%3A%20circle%20outside%3B%0A%7D%0Aul%20ul%20%7B%0Amargin%2Dbottom%3A%200%3B%0A%7D%0Apre%2C%20code%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0Aborder%2Dradius%3A%203px%3B%0Acolor%3A%20%23333%3B%0Awhite%2Dspace%3A%20pre%2Dwrap%3B%20%0A%7D%0Apre%20%7B%0Aborder%2Dradius%3A%203px%3B%0Amargin%3A%205px%200px%2010px%200px%3B%0Apadding%3A%2010px%3B%0A%7D%0Apre%3Anot%28%5Bclass%5D%29%20%7B%0Abackground%2Dcolor%3A%20%23f7f7f7%3B%0A%7D%0Acode%20%7B%0Afont%2Dfamily%3A%20Consolas%2C%20Monaco%2C%20%27Courier%20New%27%2C%20monospace%3B%0Afont%2Dsize%3A%2085%25%3B%0A%7D%0Ap%20%3E%20code%2C%20li%20%3E%20code%20%7B%0Apadding%3A%202px%200px%3B%0A%7D%0Adiv%2Efigure%20%7B%0Atext%2Dalign%3A%20center%3B%0A%7D%0Aimg%20%7B%0Abackground%2Dcolor%3A%20%23FFFFFF%3B%0Apadding%3A%202px%3B%0Aborder%3A%201px%20solid%20%23DDDDDD%3B%0Aborder%2Dradius%3A%203px%3B%0Aborder%3A%201px%20solid%20%23CCCCCC%3B%0Amargin%3A%200%205px%3B%0A%7D%0Ah1%20%7B%0Amargin%2Dtop%3A%200%3B%0Afont%2Dsize%3A%2035px%3B%0Aline%2Dheight%3A%2040px%3B%0A%7D%0Ah2%20%7B%0Aborder%2Dbottom%3A%204px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Apadding%2Dbottom%3A%202px%3B%0Afont%2Dsize%3A%20145%25%3B%0A%7D%0Ah3%20%7B%0Aborder%2Dbottom%3A%202px%20solid%20%23f7f7f7%3B%0Apadding%2Dtop%3A%2010px%3B%0Afont%2Dsize%3A%20120%25%3B%0A%7D%0Ah4%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23f7f7f7%3B%0Amargin%2Dleft%3A%208px%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Ah5%2C%20h6%20%7B%0Aborder%2Dbottom%3A%201px%20solid%20%23ccc%3B%0Afont%2Dsize%3A%20105%25%3B%0A%7D%0Aa%20%7B%0Acolor%3A%20%230033dd%3B%0Atext%2Ddecoration%3A%20none%3B%0A%7D%0Aa%3Ahover%20%7B%0Acolor%3A%20%236666ff%3B%20%7D%0Aa%3Avisited%20%7B%0Acolor%3A%20%23800080%3B%20%7D%0Aa%3Avisited%3Ahover%20%7B%0Acolor%3A%20%23BB00BB%3B%20%7D%0Aa%5Bhref%5E%3D%22http%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0Aa%5Bhref%5E%3D%22https%3A%22%5D%20%7B%0Atext%2Ddecoration%3A%20underline%3B%20%7D%0A%0Acode%20%3E%20span%2Ekw%20%7B%20color%3A%20%23555%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Edt%20%7B%20color%3A%20%23902000%3B%20%7D%20%0Acode%20%3E%20span%2Edv%20%7B%20color%3A%20%2340a070%3B%20%7D%20%0Acode%20%3E%20span%2Ebn%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Efl%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Ech%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Est%20%7B%20color%3A%20%23d14%3B%20%7D%20%0Acode%20%3E%20span%2Eco%20%7B%20color%3A%20%23888888%3B%20font%2Dstyle%3A%20italic%3B%20%7D%20%0Acode%20%3E%20span%2Eot%20%7B%20color%3A%20%23007020%3B%20%7D%20%0Acode%20%3E%20span%2Eal%20%7B%20color%3A%20%23ff0000%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%0Acode%20%3E%20span%2Efu%20%7B%20color%3A%20%23900%3B%20font%2Dweight%3A%20bold%3B%20%7D%20%20code%20%3E%20span%2Eer%20%7B%20color%3A%20%23a61717%3B%20background%2Dcolor%3A%20%23e3d2d2%3B%20%7D%20%0A" rel="stylesheet" type="text/css" />

</head>

<body>




<h1 class="title toc-ignore">Collect the SNODAS product and build a local database.</h1>
<h4 class="author"><em>James McCreight</em></h4>
<h4 class="date"><em>2017-05-01</em></h4>



<div id="background" class="section level1">
<h1>Background</h1>
<p>Pull SNODAS data and collect into a “database” of local netcdf files.</p>
</div>
<div id="setup" class="section level1">
<h1>Setup</h1>
<p>Load the rwrfhydro package.</p>
<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">library</span>(<span class="st">&quot;rwrfhydro&quot;</span>)</code></pre></div>
<p>This is the path to the directory where you want your database to be built (it should exist and be empty for the purposes of this example):</p>
<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">snodasPath &lt;-<span class="st"> '~/wrfHydroTestCases/snodas/'</span> 
<span class="cf">if</span>(<span class="kw">dir.exists</span>(snodasPath)) <span class="kw">file.remove</span>(<span class="kw">list.files</span>(snodasPath,<span class="st">'.'</span>, <span class="dt">full.names=</span><span class="ot">TRUE</span>))</code></pre></div>
<pre><code>##  [1] TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE TRUE</code></pre>
<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r"><span class="kw">dir.create</span>(snodasPath)</code></pre></div>
<pre><code>## Warning in dir.create(snodasPath): '/Users/james/wrfHydroTestCases/snodas' already exists</code></pre>
</div>
<div id="get-some-data" class="section level1">
<h1>Get some data</h1>
<p>Grab a single date (both depth and SWE) and transform to netcdf. For the sake of illustration, grab three days ago.</p>
<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">threeDaysBack &lt;-<span class="st"> </span><span class="kw">Sys.Date</span>() <span class="op">+</span><span class="st"> </span>lubridate<span class="op">::</span><span class="kw">period</span>(<span class="op">-</span><span class="dv">3</span>,<span class="st">'day'</span>)
snodasGot &lt;-<span class="st"> </span><span class="kw">GetSnodasDepthSweDate</span>(threeDaysBack, <span class="dt">outputDir=</span>snodasPath, <span class="dt">overwrite=</span><span class="ot">TRUE</span>)
<span class="cf">if</span>(snodasGot) {
  snodasList &lt;-<span class="st"> </span><span class="kw">ReadSnodasDepthSweDate</span>(threeDaysBack, <span class="dt">outputDir=</span>snodasPath)
  snodasNcFile &lt;-<span class="st"> </span><span class="kw">PutSnodasNcdf</span>(snodasList, <span class="dt">outputDir=</span>snodasPath)
  snodasNcFile
}</code></pre></div>
<pre><code>## [1] &quot;~/wrfHydroTestCases/snodas//SNODAS_20170428.nc&quot;</code></pre>
</div>
<div id="read-ncdf-and-visualize" class="section level1">
<h1>Read ncdf and visualize</h1>
<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">swe &lt;-<span class="st"> </span><span class="kw">ncdump</span>(snodasNcFile, <span class="dt">variable =</span> <span class="st">'SWE'</span>)</code></pre></div>
<pre><code>## File: ~/wrfHydroTestCases/snodas//SNODAS_20170428.nc
## ( NC_FORMAT_NETCDF4 ):
## dimensions (3):
##     Longitude = 6935 ; 
##     Latitude = 3351 ; 
##     Time = UNLIMITED ; // (1 currently)
## variables (3):
##     double Time(Time) ; 
##         Time:units = &quot;POSIXct&quot;
##         Time:long_name = &quot;Time&quot;
##     double SWE(Time,Latitude,Longitude)   (Chunking: [1265,610,1])   ; 
##         SWE:units = &quot;mm&quot; ;
##         SWE:_FillValue = &quot;-9.999&quot; ;
##         SWE:long_name = &quot;Snow water equivalent&quot; ;
##     double snowDepth(Time,Latitude,Longitude)   (Chunking: [1265,610,1])   ; 
##         snowDepth:units = &quot;mm&quot; ;
##         snowDepth:_FillValue = &quot;-9.999&quot; ;
##         snowDepth:long_name = &quot;Snow depth&quot; ;
## 
## // global attributes (2):
##     :Time = &quot;2012-07-05_00:00:00&quot;
##     :POSIXct Origin = &quot;1970-01-01 00:00.00 UTC&quot;</code></pre>
<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">## Filter outliers.
swe[swe<span class="op">&gt;</span><span class="kw">quantile</span>(<span class="kw">as.vector</span>(swe), .<span class="dv">999</span>, <span class="dt">na.rm=</span><span class="ot">TRUE</span>)] &lt;-<span class="st"> </span><span class="ot">NA</span>
<span class="kw">image</span>(swe)</code></pre></div>
<p><img src="" width="700" height="525" /></p>
</div>
<div id="update" class="section level1">
<h1>Update</h1>
<p>The above can be repeated over multiple dates one might want to get, for example from three days ago until now</p>
<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">datesWanted &lt;-<span class="st"> </span><span class="kw">seq</span>(threeDaysBack, <span class="kw">Sys.Date</span>(), <span class="dt">by =</span> <span class="st">'days'</span>)
datesWanted    </code></pre></div>
<pre><code>## [1] &quot;2017-04-28&quot; &quot;2017-04-29&quot; &quot;2017-04-30&quot; &quot;2017-05-01&quot;</code></pre>
<p>Encapsulate the above in a function with some file existance checking and return summary.</p>
<div class="sourceCode"><pre class="sourceCode r"><code class="sourceCode r">UpdateSnodas &lt;-<span class="st"> </span><span class="cf">function</span>(POSIXct, <span class="dt">outPath=</span><span class="st">'.'</span>) {
  ## check if we already processed this date/POSIXct, if we didnt process, we'll
  ## download again and process it. (note we could have downloaded but not processed 
  ## so it might not be efficient). 
  file &lt;-<span class="st"> </span><span class="kw">paste0</span>(outPath, <span class="st">'SNODAS_'</span>,<span class="kw">format</span>(POSIXct,<span class="st">'%Y%m%d'</span>),<span class="st">'.nc'</span>)
  processed &lt;-<span class="st"> </span><span class="kw">file.exists</span>(file)
  <span class="cf">if</span>(processed) 
    <span class="kw">return</span>(<span class="kw">data.frame</span>(<span class="dt">date=</span>POSIXct, <span class="dt">snodasGot=</span><span class="ot">FALSE</span>, <span class="dt">ncdfFile=</span>file))
  
  snodasGot &lt;-<span class="st"> </span><span class="kw">GetSnodasDepthSweDate</span>(POSIXct, <span class="dt">outputDir=</span>outPath)
  <span class="cf">if</span>(snodasGot) {
    snodasList &lt;-<span class="st"> </span><span class="kw">ReadSnodasDepthSweDate</span>(POSIXct, <span class="dt">outputDir=</span>outPath)
    ncdfFile &lt;-<span class="st"> </span><span class="kw">PutSnodasNcdf</span>(snodasList, <span class="dt">outputDir=</span>outPath)
  } <span class="cf">else</span> ncdfFile &lt;-<span class="st"> ''</span>
  <span class="kw">data.frame</span>(<span class="dt">date=</span>POSIXct, <span class="dt">snodasGot=</span>snodasGot, <span class="dt">ncdfFile=</span>ncdfFile)
}

update &lt;-<span class="st"> </span>plyr<span class="op">::</span><span class="kw">ldply</span>(<span class="kw">NamedList</span>(datesWanted), UpdateSnodas, <span class="dt">outPath=</span>snodasPath)
update</code></pre></div>
<pre><code>##          .id       date snodasGot                                       ncdfFile
## 1 2017-04-28 2017-04-28     FALSE  ~/wrfHydroTestCases/snodas/SNODAS_20170428.nc
## 2 2017-04-29 2017-04-29      TRUE ~/wrfHydroTestCases/snodas//SNODAS_20170429.nc
## 3 2017-04-30 2017-04-30      TRUE ~/wrfHydroTestCases/snodas//SNODAS_20170430.nc
## 4 2017-05-01 2017-05-01      TRUE ~/wrfHydroTestCases/snodas//SNODAS_20170501.nc</code></pre>
<p>Shows that the first file wasnt “got” - that’s because it was already in place.</p>
</div>



<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
  (function () {
    var script = document.createElement("script");
    script.type = "text/javascript";
    script.src  = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
    document.getElementsByTagName("head")[0].appendChild(script);
  })();
</script>

</body>
</html>
